/**
 *
 * Copyright Pact Lab of H.I.T.
 *
 * Designed and Implemented by Grid Researching Group, 
 * Pact Lab, Harbin
 * 
 * This Project is part of the national 973 Project:
 * Internet Based Virtual Computing Environment
 *
 * http://pact518.hit.edu.cn
 * 
 * Author:       Meteor <meteorlxk@gmail.com> 
 * Copyright:    pact518 
 * Version:      1.0
 * Created:      2009-5-2 
 * LastModified: 2009-5-2
 */
package edu.hit.pact.pgse.crawler.blog;

import java.net.MalformedURLException;
import java.net.URL;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import edu.hit.pact.pgse.bean.RawWebPage;
import edu.hit.pact.pgse.bean.RunnableTask;
import edu.hit.pact.pgse.bean.TaskPieceProperty;
import edu.hit.pact.pgse.crawler.Crawler;
import edu.hit.pact.pgse.crawler.finish.FinishedReportedProcessor;

/**
 * @author meteorlxk
 *
 */
public class BlogCrawler extends Crawler{
	
	public BlogCrawler(RunnableTask runnableTask, FinishedReportedProcessor finishedReporter, boolean ifSend, boolean ifStore) {
		super(runnableTask, finishedReporter, LogFactory.getLog(BlogCrawler.class), ifSend, ifStore);
	}

	public void start(){

//		RawWebPage page = null;
		String homePage = pieceProperty.getPiece().getHomePage();
//		URL homePageUrl = null;
//		try {
//			homePageUrl = new URL(homePage);
//		} catch (MalformedURLException e) {
//			// TODO Auto-generated catch block
//			e.printStackTrace();
//		}
////		pieceProperty.incrementCleanUrlQueueNum();
//
//		page = this.fetcher.fetchURL(homePageUrl, 2, null, null, null);
//		if (page == null){
//			pieceProperty.incrementFailedPageNum();
////			pieceProperty.decrementCleanUrlQueueNum();
//			return ;
//		}
//		
//		pieceProperty.incrementSucceededPageNum();
////		pieceProperty.decrementCleanUrlQueueNum();
//		this.fetchedPageQueue.add(page);
//		this.store.storeWebPage(page);
//		
//		this.pieceProperty.setStatus(TaskPieceProperty.FINISHED);
//		finishedReporter.start();
//		logger.info("blog crawler is dying!");
		
		Log plogger = LogFactory.getLog("blogId");
		String pattern = "0000000";
		int i = 149311;
		java.text.DecimalFormat df = new java.text.DecimalFormat(pattern);
		String url = "http://blog.tianya.cn/blogger/blog_main.asp?BlogID=";
		URL compareUrl = null;
		try {
			compareUrl = new URL("http://blog.tianya.cn/blogger/blog_main.asp?BlogID=0000040");
		} catch (MalformedURLException e1) {
			
			e1.printStackTrace();
		}
		RawWebPage comparePage = this.fetcher.fetchURL(compareUrl, 2, null, null, null);
		
		this.store.storeWebPage(comparePage, "test","-1",true);
		
		//System.out.println(comparePage.getText());
		
		while(i++ < 2000000){
			String tmpUrl = url+df.format(i);
			//System.out.println(tmpUrl);
			URL realUrl = null;
			try {
				realUrl = new URL(tmpUrl);
			} catch (MalformedURLException e) {
				e.printStackTrace();
			}
			
			RawWebPage page = this.fetcher.fetchURL(realUrl, 2, null, null, null);
			String txt = page.getText();
			boolean flag = false;
			for(int x=1;x<14;x++){
				if(txt.equals("redirect to error.asp?msgid="+x)){
					flag = true;
					break;
				}
			}
			if(flag){
				continue;
			}
			
			plogger.info(tmpUrl);
			this.store.storeWebPage(page, "test", i+"", true);
			
		}
		
	}
	
	public void deactivate(){
		logger.info("closing blog crawler");
		super.deactivate();
	}
}
